﻿using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Elderos.TextSearchers
{
	public class RegexTokenizer : ITokenizer
	{
		public RegexTokenizer(bool excludeWhitespaces)
		{
			ExcludeWhitespaces = excludeWhitespaces;
		}

		public bool ExcludeWhitespaces { get; private set; }

		public static Regex TokenizationRegex = new Regex(
			@"[\w-]+|[^\w-]",
			RegexOptions.Compiled
			);

		#region Implementation of ITokenizer

		public string[] Tokenize(string text)
		{
			if (String.IsNullOrEmpty(text))
				throw new ArgumentException("text");

			var strings = TokenizationRegex.Matches(text)
				.Cast<Match>()
				.Select(x => x.Value);

			if (ExcludeWhitespaces)
				strings = strings
					.Select(x => x.Trim())
					.Where(x => !String.IsNullOrWhiteSpace(x));
			return strings.ToArray();
		}

		#endregion
	}
}
